#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @File  : jieba_gov_report.py
# @Author: evenvi
# @Date  : 19-4-28
# @Desc  : 使用Jieba对2019政府工作报告分词
import codecs
import uniout # 中文显示


# 读取待分词文本数据
def get_content(path, encoding='gbk'):
    with codecs.open(path, 'r', encoding=encoding
            , errors='ignore') as f:
        content = ''
        for l in f:
            l = l.strip()
            content += l
        return content

# 定义高频词函数
def get_TF(words, topK=10):
    tf_dic = {}
    for w in words:
        tf_dic[w] = tf_dic.get(w, 0) + 1
    return sorted(tf_dic.items(), key=lambda x:x[1], reverse=True)[:topK]

def main():
    import jieba

    reportContent = get_content('./data/reports/2019gov.txt', encoding='utf-8')

    split_words = list(jieba.cut(reportContent))
    print('样本之一：'+ reportContent.encode('utf-8') + '\r\n')
    print('样本分词效果:' + '/'.join(split_words).encode('utf-8')+ '\r\n')
    print('样本的topK(10)词语' + str(get_TF(split_words))+ '\r\n')

if __name__ == '__main__':
    main()

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @File  : jieba_gov_report.py
# @Author: evenvi
# @Date  : 19-4-28
# @Desc  : 使用Jieba对2019政府工作报告分词
import codecs
import uniout # 中文显示


# 读取待分词文本数据
def get_content(path, encoding='gbk'):
    with codecs.open(path, 'r', encoding=encoding
            , errors='ignore') as f:
        content = ''
        for l in f:
            l = l.strip()
            content += l
        return content

# 定义高频词函数
def get_TF(words, topK=10):
    tf_dic = {}
    for w in words:
        tf_dic[w] = tf_dic.get(w, 0) + 1
    return sorted(tf_dic.items(), key=lambda x:x[1], reverse=True)[:topK]

def stop_words(path):
    with codecs.open(path, 'r', encoding='utf-8' , errors='ignore') as f:
        return [l.strip() for l in f]

def main():
    import jieba

    reportContent = get_content('./data/reports/2019gov.txt', encoding='utf-8')

    # split_words = list(jieba.cut(reportContent))
    split_words = [x for x in jieba.cut(reportContent) if x not in stop_words('./data/stop_words.utf8')]
    print('样本之一：'+ reportContent.encode('utf-8') + '\r\n')
    print('样本分词效果:' + '/'.join(split_words).encode('utf-8')+ '\r\n')
    print('样本的topK(10)词语' + str(get_TF(split_words))+ '\r\n')

if __name__ == '__main__':
    main()

